const path = require("path");
const { createWriteStream } = require("fs");
const { promisify } = require("util");
const pipeline = promisify(require("stream").pipeline);
const crypto = require("crypto");
const axios = require("axios");
const cheerio = require("cheerio");
const mysql = require("mysql2/promise");
const fs = require("fs").promises;
const moment = require("moment");

// 配置中心
const CONFIG = {
  database: {
    host: "localhost",
    user: "root",
    password: "oa1JCczD3jLLlE7M",
    database: "chen_db_2004",
    waitForConnections: true,
    connectionLimit: 10,
    queueLimit: 0,
  },
  headers: {
    "User-Agent":
      "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36",
    Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
    "Accept-Language": "zh-CN,zh;q=0.9",
  },
  concurrency: 3, // 并发控制
  image: {
    dir: path.join(__dirname, "images"),
    timeout: 15000, // 15秒超时
    retries: 3, // 下载重试次数
  },
};

// 初始化数据库连接池
const pool = mysql.createPool(CONFIG.database);

async function downloadImage(url, newsId) {
  if (!url) return null;

  const hash = crypto.createHash("md5").update(url).digest("hex");
  const ext = path.extname(new URL(url).pathname).slice(0, 5) || ".jpg"; // 限制扩展名长度
  const filename = `${newsId}_${hash}${ext}`.replace(/[/\\?*:|"<>]/g, "_");
  const filepath = path.join(CONFIG.image.dir, filename);

  for (let attempt = 1; attempt <= CONFIG.image.retries; attempt++) {
    try {
      const response = await axios({
        url,
        timeout: CONFIG.image.timeout,
        responseType: "stream",
        headers: CONFIG.headers,
      });

      await fs.mkdir(CONFIG.image.dir, { recursive: true });
      await pipeline(response.data, createWriteStream(filepath));
      return path.join("/images", filename);
    } catch (error) {
      if (attempt === CONFIG.image.retries) {
        console.error(`图片下载失败: ${url}`, error.message);
        return null;
      }
      await new Promise((res) => setTimeout(res, 2000 * attempt)); // 指数退避
    }
  }
}

function parsePublishTime(text) {
  const formats = [
    "YYYY年MM月DD日 HH:mm",
    "YYYY-MM-DD HH:mm:ss",
    "YYYY-MM-DDTHH:mm:ssZ", // ISO格式
    "X", // 时间戳
  ];

  const time = moment(text, formats);
  return time.isValid()
    ? time.format("YYYY-MM-DD HH:mm:ss")
    : moment().format("YYYY-MM-DD HH:mm:ss");
}

async function processNewsItem(item, connection) {
  try {
    // 获取详情页内容
    const { data: detailHtml } = await axios.get(item.link, {
      headers: CONFIG.headers,
      timeout: 10000,
    });
    const $d = cheerio.load(detailHtml);

    // 内容提取
    const content = $d('[data-type="rticle"] p, .article-body p, .content p')
      .text()
      .trim()
      .substring(0, 5000);

    // 时间解析
    const timeText =
      $d(
        '.date-source span.date, .time-source, [property="article:published_time"]'
      )
        .first()
        .attr("content") || $d(".date").text();
    const publishTime = parsePublishTime(timeText);

    // 图片处理
    const imgSrc =
      $d(".img_wrapper img").attr("src") ||
      $d(".img_wrapper img").attr("data-src") ||
      $d('[property="og:image"]').attr("content");
    const imageUrl = new URL(imgSrc, item.link).href;

    // 先插入数据库获取新闻ID
    const [result] = await connection.execute(
      `INSERT INTO news_items(title, content, publish_time) VALUES (?, ?, ?)`,
      [item.title.substring(0, 255), content, publishTime]
    );

    // 使用数据库自增ID下载图片
    const imagePath = await downloadImage(imageUrl, result.insertId);

    // 更新图片路径
    await connection.execute(
      `UPDATE news_items SET image_url = ? WHERE id = ?`,
      [imagePath, result.insertId]
    );

    console.log(`成功保存：${item.title.substring(0, 50)}`);
    return true;
  } catch (error) {
    console.error(`处理失败 ${item.title}:`, error.message);
    return false;
  }
}

async function scrapeNews() {
  const baseUrl = "https://news.sina.com.cn";
  let connection;

  try {
    connection = await pool.getConnection();
    const { data } = await axios.get(baseUrl, {
      headers: CONFIG.headers,
      timeout: 8000,
    });
    const $ = cheerio.load(data);

    // 改进的选择器
    const newsItems = [];
    $('[data-client="headline"], .news-item, .feed-card-item').each((i, el) => {
      const $el = $(el);
      const title = $el.find("a").text().trim();
      const href = $el.find("a").attr("href");

      if (title && href) {
        newsItems.push({
          title,
          link: new URL(href, baseUrl).href,
        });
      }
    });

    if (newsItems.length === 0) throw new Error("未找到有效新闻条目");

    // 并发控制
    for (let i = 0; i < newsItems.length; i += CONFIG.concurrency) {
      const chunk = newsItems.slice(i, i + CONFIG.concurrency);
      await Promise.all(chunk.map((item) => processNewsItem(item, connection)));
      console.log(
        `已完成 ${Math.min(i + CONFIG.concurrency, newsItems.length)}/${
          newsItems.length
        }`
      );
    }
  } catch (error) {
    console.error("爬取失败:", error);
  } finally {
    if (connection) connection.release();
    // 保持连接池开启以便后续操作
    // await pool.end(); // 程序完全退出时再关闭
  }
}

// 启动爬虫
scrapeNews()
  .then(() => process.exit(0))
  .catch(() => process.exit(1));
